In [2]:
%matplotlib qt4
from __future__ import division

from models import tools, optimize, models, filters
from models.tests import PerformanceTest

import numpy as np
import pandas as pd
import sklearn as sk
import matplotlib.pyplot as plt

In [4]:
data = tools.load_data(offset=5e6, limit=1e6)
data = data[filters.place_type(data, 'river') & filters.for_staircase(data)]
print len(data)


109394

In [3]:
data = tools.add_last_trial_age(data)

In [19]:
only_old = lambda d: d[d['last_trial_age'] > 60*60*6]
only_new = lambda d: d[d['last_trial_age'] < 60*60*6]

Elo Model


In [5]:
elo_test = PerformanceTest(models.EloModel(), data, split_data=True)
elo_test.run()
elo_test.results['train']


Out[5]:
RMSE: 0.423472181742
AUC: 0.708818093631
LL: -12401.6142926
OFF: 0.00353807736108
CORRECT: 16978
ACCURACY: 0.73536036036
Set Size: 23088

Elo With Response Time


In [6]:
elot_test = PerformanceTest(models.EloResponseTime(), data, split_data=True)
elot_test.run()
elot_test.results['train']


Out[6]:
RMSE: 0.428191222313
AUC: 0.703571244911
LL: -12709.1073088
OFF: -0.0139681485231
CORRECT: 16865
ACCURACY: 0.730466042966
Set Size: 23088

PFA


In [5]:
pfa = models.PFAModel(models.EloModel(), gamma=1.545, delta=-0.21)
pfa_test = PerformanceTest(pfa, data).run()
pfa_test.get_results()


Out[5]:
RMSE: 0.376769079342
AUC: 0.748113754892
LL: -66675.5597371
OFF: 0.0185287086654
CORRECT: 89404
ACCURACY: 0.817266029215
Set Size: 109394

In [20]:
pfa_test.get_results(only_new)


Out[20]:
RMSE: 0.354397301148
AUC: 0.732515794771
LL: -43559.4722765
OFF: 0.0157483008845
CORRECT: 70419
ACCURACY: 0.832149651986
Set Size: 84623

In [21]:
pfa_test.get_results(only_old)


Out[21]:
RMSE: 0.366541495727
AUC: 0.726653077167
LL: -16119.9074797
OFF: 0.0700024341793
CORRECT: 22114
ACCURACY: 0.831947631767
Set Size: 26581

PFA/E


In [6]:
pfae = models.PFAExt(models.EloModel(), gamma=2.92, delta=-0.759)
pfae_test = PerformanceTest(pfae, data).run()
pfae_test.get_results()


Out[6]:
RMSE: 0.351961960655
AUC: 0.798228055245
LL: -43511.7937805
OFF: -0.0153600124807
CORRECT: 90916
ACCURACY: 0.831087628206
Set Size: 109394

In [22]:
pfae_test.get_results(only_new)


Out[22]:
RMSE: 0.336695163302
AUC: 0.786101580598
LL: -31061.6516397
OFF: -0.0364471491907
CORRECT: 71279
ACCURACY: 0.842312373705
Set Size: 84623

In [24]:
pfae_test.get_results(only_old)


Out[24]:
RMSE: 0.349051935629
AUC: 0.752187080372
LL: -10574.8068286
OFF: 0.0255620929098
CORRECT: 22334
ACCURACY: 0.840224220308
Set Size: 26581

PFA/G


In [9]:
pfag = models.PFAGong(models.EloModel(), gamma=2.024, delta=0.066, decay=0.434)
pfag_test = PerformanceTest(pfag, data).run()
pfag_test.get_results()


Out[9]:
RMSE: 0.356887563781
AUC: 0.781087497326
LL: -47487.8629404
OFF: 0.0378666367104
CORRECT: 91129
ACCURACY: 0.83303471854
Set Size: 109394

In [25]:
pfag_test.get_results(only_new)


Out[25]:
RMSE: 0.341157607473
AUC: 0.765819355989
LL: -32775.9443002
OFF: 0.0138780532659
CORRECT: 71274
ACCURACY: 0.842253288113
Set Size: 84623

In [26]:
pfag_test.get_results(only_old)


Out[26]:
RMSE: 0.35695634306
AUC: 0.753974053078
LL: -11782.9196501
OFF: 0.0731836694598
CORRECT: 22315
ACCURACY: 0.839509424025
Set Size: 26581

PFA/E/T

Pow


In [35]:
pfaet = models.PFAExtTiming(models.EloModel(), gamma=2.004, delta=-0.713,
                            time_effect_fun='pow', a=2.931, c=0.27)
pfaet_test = PerformanceTest(pfaet, data).run()
pfaet_test.get_results()


Out[35]:
RMSE: 0.340610555244
AUC: 0.806426282763
LL: -54082.4963446
OFF: 0.00152109514334
CORRECT: 121341
ACCURACY: 0.841547146781
Set Size: 144188

In [36]:
pfaet_test.get_results(only_new)


Out[36]:
RMSE: 0.330289864694
AUC: 0.784095497917
LL: -30159.6216451
OFF: 0.00525999303916
CORRECT: 72066
ACCURACY: 0.851612445789
Set Size: 84623

In [37]:
pfaet_test.get_results(only_old)


Out[37]:
RMSE: 0.341718412215
AUC: 0.776883914379
LL: -10056.0850104
OFF: -0.00999920791694
CORRECT: 22378
ACCURACY: 0.841879538016
Set Size: 26581

Log


In [38]:
pfaet = models.PFAExtTiming(models.EloModel(), gamma=1.906, delta=-0.806,
                            time_effect_fun='log', a=1.789, c=0.128)
pfaet_test = PerformanceTest(pfaet, data).run()
pfaet_test.get_results()


Out[38]:
RMSE: 0.340689226279
AUC: 0.806618196313
LL: -54088.4268855
OFF: -0.00167219711391
CORRECT: 121301
ACCURACY: 0.841269731184
Set Size: 144188

In [39]:
pfaet_test.get_results(only_new)


Out[39]:
RMSE: 0.3303492141
AUC: 0.78401170611
LL: -30161.0209555
OFF: 0.00268885024601
CORRECT: 72050
ACCURACY: 0.851423371897
Set Size: 84623

In [40]:
pfaet_test.get_results(only_old)


Out[40]:
RMSE: 0.341961099325
AUC: 0.778680616398
LL: -10060.6162409
OFF: -0.0191356816163
CORRECT: 22354
ACCURACY: 0.840976637448
Set Size: 26581

Exp


In [10]:
pfaet = models.PFAExtTiming(models.EloModel(), gamma=2.006, delta=-0.757,
                            time_effect_fun='exp', a=1.005, c=0.009)
pfaet_test = PerformanceTest(pfaet, data).run()
pfaet_test.get_results()


Out[10]:
RMSE: 0.341023057373
AUC: 0.808491437126
LL: -54145.6164025
OFF: -0.00988678960385
CORRECT: 121099
ACCURACY: 0.839868782423
Set Size: 144188

In [27]:
pfaet_test.get_results(only_new)


Out[27]:
RMSE: 0.330511174497
AUC: 0.78680964956
LL: -30136.5564007
OFF: -0.00862470783505
CORRECT: 71886
ACCURACY: 0.849485364499
Set Size: 84623

In [28]:
pfaet_test.get_results(only_old)


Out[28]:
RMSE: 0.343265390481
AUC: 0.776443230122
LL: -10142.2703127
OFF: -0.0276778145852
CORRECT: 22316
ACCURACY: 0.839547044882
Set Size: 26581

Staircase


In [11]:
staircase = {
    (0, 60): 1.43,
    (60, 90): 1.17,
    (90, 150): 1.01,
    (150, 300): 0.93,
    (300, 600): 0.82,
    (600, 60*30): 0.78,
    (60*30, 60*60*3): 0.76,
    (60*60*3, 60*60*24): 0.63,
    (60*60*24, 60*60*24*5): 0.42,
    (60*60*24*5, np.inf): 0.12,
}

pfaest = models.PFAExtStaircase(models.EloModel(), staircase=staircase,
                                gamma=1.814, delta=-0.827)
pfaest_test = PerformanceTest(pfaest, data).run()
pfaest_test.get_results()


Out[11]:
RMSE: 0.340814661115
AUC: 0.807544282931
LL: -54114.2313641
OFF: -0.00774427081198
CORRECT: 121218
ACCURACY: 0.840694093822
Set Size: 144188

In [29]:
pfaest_test.get_results(only_new)


Out[29]:
RMSE: 0.330705025338
AUC: 0.78361749371
LL: -30212.9076876
OFF: -0.0104702573791
CORRECT: 71939
ACCURACY: 0.850111671768
Set Size: 84623

In [30]:
pfaest_test.get_results(only_old)


Out[30]:
RMSE: 0.341543975675
AUC: 0.778966847822
LL: -10034.5339873
OFF: -0.0101802998706
CORRECT: 22382
ACCURACY: 0.842030021444
Set Size: 26581

In [11]:
x = [0, 40.7, 74.7, 115.7, 205.6, 415.1, 1023.5, 4635.0, 48792.6, 198540.1, 621875.5, 10e10]
y = [1.2, 1.001, 0.746, 0.609, 0.481, 0.387, 0.38, 0.272, 0.166, 0.153, -0.164, -0.2]

staircase2 = tools.connect_points(zip(x, y))
pfaest2 = models.PFAExtTiming(models.EloModel(), time_effect_fun=staircase2,
                              gamma=2.554, delta=-0.766)
pfaest2_test = PerformanceTest(pfaest2, data).run()
pfaest2_test.get_results()


Out[11]:
RMSE: 0.347675069547
AUC: 0.802913276382
LL: -42511.58677
OFF: 0.00290940747907
CORRECT: 91356
ACCURACY: 0.835109786643
Set Size: 109394

In [33]:
pfaest2_test.get_results(only_new)


Out[33]:
RMSE: 0.330472283244
AUC: 0.783973003638
LL: -30178.0802675
OFF: -0.00872058989969
CORRECT: 71957
ACCURACY: 0.850324379897
Set Size: 84623

In [34]:
pfaest2_test.get_results(only_old)


Out[34]:
RMSE: 0.3418059752
AUC: 0.778321976835
LL: -10045.4933616
OFF: -0.0100427380118
CORRECT: 22384
ACCURACY: 0.842105263158
Set Size: 26581

PFA/G/T

Pow


In [10]:
pfagt = models.PFATiming(models.EloModel(),
                         time_effect_good='pow', a=3.196, c=0.21,
                         time_effect_bad='pow', b=4.705, d=0.695)
pfagt_test = PerformanceTest(pfagt, data).run()
pfagt_test.get_results()


Out[10]:
RMSE: 0.359027722712
AUC: 0.7716510051
LL: -48976.4274892
OFF: 0.0266485026234
CORRECT: 90722
ACCURACY: 0.829314221987
Set Size: 109394

In [45]:
pfagt_test.get_results(only_new)0.363359462169


Out[45]:
RMSE: 0.342350961066
AUC: 0.755554559652
LL: -35584.1280647
OFF: 0.0225048367109
CORRECT: 71292
ACCURACY: 0.842465996242
Set Size: 84623

In [46]:
pfagt_test.get_results(only_old)


Out[46]:
RMSE: 0.351600464616
AUC: 0.754879028574
LL: -10676.7078492
OFF: -0.0183476976578
CORRECT: 22049
ACCURACY: 0.829502276062
Set Size: 26581

Log


In [47]:
pfagt = models.PFATiming(models.EloModel(),
                         time_effect_good='log', a=1.669, c=0.102,
                         time_effect_bad='log', b=0.914, d=0.098)
pfagt_test = PerformanceTest(pfagt, data).run()
pfagt_test.get_results()


Out[47]:
RMSE: 0.351595326154
AUC: 0.775257897025
LL: -63083.6694556
OFF: 0.0180259204946
CORRECT: 119876
ACCURACY: 0.835291330463
Set Size: 143514

In [48]:
pfagt_test.get_results(only_new)


Out[48]:
RMSE: 0.345969738301
AUC: 0.741354472004
LL: -38293.320991
OFF: 0.0375190317222
CORRECT: 70891
ACCURACY: 0.844453179907
Set Size: 83949

In [49]:
pfagt_test.get_results(only_old)


Out[49]:
RMSE: 0.352364106136
AUC: 0.770791164391
LL: -10923.5587755
OFF: -0.0226744464621
CORRECT: 22088
ACCURACY: 0.830969489485
Set Size: 26581

Exp


In [8]:
pfagt = models.PFATiming(models.EloModel(),
                         time_effect_good='exp', a=1.2, c=0.0017,
                         time_effect_bad='exp', b=1.496, d=0.012)
pfagt_test = PerformanceTest(pfagt, data).run()
pfagt_test.get_results()


Out[8]:
RMSE: 0.381497383305
AUC: 0.727927046887
LL: -4930.92175252
OFF: 0.0421144648418
CORRECT: 7377
ACCURACY: 0.81235546746
Set Size: 9081

In [53]:
pfagt_test.get_results(only_new)


Out[53]:
RMSE: 0.344712541379
AUC: 0.740277853289
LL: -37075.9728314
OFF: 0.0239399120484
CORRECT: 71325
ACCURACY: 0.842855961145
Set Size: 84623

In [54]:
pfagt_test.get_results(only_old)


Out[54]:
RMSE: 0.35210576888
AUC: 0.750118153429
LL: -10938.7822215
OFF: -0.00673720396386
CORRECT: 22087
ACCURACY: 0.830931868628
Set Size: 26581

PFA/E/RT


In [14]:
pfart = models.PFAResponseTime(models.EloModel(), gamma=1.299, delta=-0.951, zeta=1.916)
pfart_test = PerformanceTest(pfart, data).run()
pfart_test.get_results()


Out[14]:
RMSE: 0.352294694451
AUC: 0.793839944944
LL: -44757.9270573
OFF: 0.00101258640877
CORRECT: 90668
ACCURACY: 0.828820593451
Set Size: 109394

In [41]:
pfart_test.get_results(only_new)0.352750053521


Out[41]:
RMSE: 0.336790457866
AUC: 0.785213286488
LL: -32161.6977155
OFF: -0.0183581208724
CORRECT: 71009
ACCURACY: 0.83912175177
Set Size: 84623

In [42]:
pfart_test.get_results(only_old)


Out[42]:
RMSE: 0.348672299028
AUC: 0.768197627565
LL: -11308.0014656
OFF: 0.0502793256465
CORRECT: 22356
ACCURACY: 0.841051879162
Set Size: 26581

In [ ]: